<?php
abstract class Base_Extractor
{
  /**
   * Store indexation uri
   * @var string
   */
  public $url;
  /**
   * Store dom of current pgae
   * @var object
   */
  private $dom;
  /**
   * Store object of current curl 
   * @var object
   */
  private $curl;
  /**
   * Variable with response headers
   * @var array
   */
  private $headers;
  /**
   * Variable with server result
   * @var string
   */
  private $header;
  
  function __construct()
  {
    if(!class_exists ("DOMDocument", false) )
    {
      throw new Exception ("Class 'DOMDocument' does not exists.");
    }
    
    if(!class_exists ("DOMXPath", false) )
    {
      throw new Exception ("Class 'DOMXPath' does not exists.");
    }
    
    if(!function_exists("curl_exec"))
    {
      throw new Exception ("Curl extention does not exists.");
    }
    
    $this->curl = curl_init();
  }
  /**
   * Read headers and store to common array 
   * @param resourse $curl
   * @param string $header
   */
  protected function doReadHeader($curl, $header)
  {
    $header_length = mb_strlen($header);
    $header        = trim($header);
    
    if(mb_strlen(trim($header)))
    {
      $delimiter_position = strpos($header, ":");
      
      if($delimiter_position === false)
      {
        $this->header = $header;
        
        return $header_length;
      }
      
      $key    = trim(substr($header, 0, $delimiter_position));
      $value  = trim(substr($header, $delimiter_position + 1));
      
      $this->headers[$key] = $value;
    }
    return $header_length;
  }  
  /**
   * Make unique link format
   * @param string $parent
   * @param string $link
   */
  public function doNormalizeLink($parent, $link)
  {
    $parsed_parent  = parse_url(urldecode($parent));
    $parsed_link    = parse_url(urldecode($link));
    
    if(! $parsed_parent || ! $parsed_link)
    {
      return null;
    }
    
    $parent_keys_to_remove = array(
      "user"      => "", 
      "pass"      => "", 
      "path"      => "", 
      "query"     => "", 
      "fragment"  => ""
    );
    
    $parsed_parent = array_diff_key($parsed_parent, $parent_keys_to_remove);
    
    $link_keys_to_remove = array(
      "user"      => "", 
      "pass"      => "", 
      "fragment"  => ""
    );
    
    $parsed_link = array_diff_key($parsed_link, $link_keys_to_remove);
    
    if(! $parsed_parent || ! $parsed_link)
    {
      return null;
    }
    
    return urldecode(http_build_url(array_merge($parsed_parent, $parsed_link)));
  }
 /**
   * Check for permanently moved page
   * @return boolean
   */
  public function isMovedPermanently()
  {
    $statuses = array("301", "302");
    
    foreach($statuses as $status)
    {
      if(strpos($this->header, $status) !== false)
      {
        return true;
      }
    }
    
    return false;
  }  
  /**
   * Check is link is inner
   * @param string $parent
   * @param string $link
   */
  public function isInnerLink($parent, $link)
  {
    $parsed_parent  = parse_url($parent);
    $parsed_link    = parse_url($link);
    
    if(! isset($parsed_link["host"]))
    {
      return true;
    }
    
    if($parsed_parent["host"] == $parsed_link["host"])
    {
      if(isset($parsed_parent["scheme"]) && isset($parsed_link["scheme"]))
      {
        if($parsed_parent["scheme"] == $parsed_link["scheme"])
        {
          return true;
        }
      }
    }
    return false;
  }
  
  public function getDomain($url = null)
  {
    $parsed_url = parse_url(urldecode($url));
    
    if(is_array($parsed_url) && count($parsed_url))
    {
      if(isset($parsed_url["host"]))
      {
        return $parsed_url["host"];
      }
    }
    return null;
  }
    
  public function getLinksFromDoom ($url)
  {
    $xpath = new DOMXPath($this->getContent($url));
    
    return $xpath->evaluate("//a"); //get all a tags
  }  
  /**
   * Method to get inner links
   * @param $url
   */
  public function getInnerLinks($url)
  {
    $hrefs = $this->getLinksFromDoom($url);
    
    $links = array();
    
    for($i = 0; $i < $hrefs->length; $i ++)
    {
      $href = $hrefs->item($i); //select an a tag
      $link = $this->doNormalizeLink($url, $href->getAttribute('href'));
      
      if(! $this->isInnerLink($url, $link))
      {
        continue;
      }
      
      $links[] = $link;
    }
    return $links;
  }
  /**
   * Method to get outer links
   * @param $url
   */
  public function getOuterLinks($url)
  {
    $hrefs = $this->getLinksFromDoom($url);
    $links = array();
    
    for($i = 0; $i < $hrefs->length; $i ++)
    {
      $href = $hrefs->item($i); //select an a tag
      $link = $this->doNormalizeLink($url, $href->getAttribute('href'));
      
      if($this->isInnerLink($url, $link))
      {
        continue;
      }
      
      $links[] = $link;
    }
    return $links;
  }
  /**
   * Method to get needed header value
   * @param $header
   */
  public function getHeader($header)
  {
    if(! isset($this->headers))
    {
      throw new Exception("Request for unknown header!");
    }
    return $this->headers[$header];
  }   
  /**
   * Get current page content
   * @param $url
   */
  public function getContent($url)
  {
    if($this->url == $url)
    {
      if(is_object($this->dom))
      {
        return $this->dom;
      }
    }
    
    $this->url    = $url;
    $options_curl = array();
    
    $options_curl[CURLOPT_URL]            = $url;
    $options_curl[CURLOPT_RETURNTRANSFER] = true;
    $options_curl[CURLOPT_HEADERFUNCTION] = array($this, 'doReadHeader');
    $options_curl[CURLOPT_USERAGENT]      = 'Googlebot/2.1 (+http://www.google.com/bot.html)';
    
    curl_setopt_array($this->curl, $options_curl);
    
    $content = curl_exec($this->curl);
    
    if(! mb_strlen($content))
    {
      if($this->isMovedPermanently())
      {
        $options_curl[CURLOPT_URL] = $this->getHeader("Location");
        
        curl_setopt_array($this->curl, $options_curl);
        
        $content = curl_exec($this->curl);
      }
    }
    
    file_put_contents("content.txt", $content);
    
    $this->dom = new DOMDocument();
    //        $tidy      = new tidy();        
    //        @$this->dom->loadHTML($tidy->repairString($content));
    @$this->dom->loadHTML($content);
    
    return $this->dom;
  }
}
?>